{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4embtkV0pNxM"
   },
   "source": [
    "# Deep Learning\n",
    "\n",
    "## preprocessing training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "input_file_dir = \"/Users/ninglee/Documents/IntrutionDection/datasets\"\n",
    "train_file_name = \"kddcup.data_10_percent.txt\"\n",
    "test_file_name = \"corrected.txt\"\n",
    "header_file_name = \"header.txt\"\n",
    "train_files = os.path.join(input_file_dir, train_file_name)\n",
    "test_files = os.path.join(input_file_dir, test_file_name)\n",
    "header_files = os.path.join(input_file_dir, header_file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(header_files, 'r') as f:\n",
    "    header = f.readline().strip().split(',')\n",
    "train_dataset = pd.read_csv(train_files)\n",
    "test_dataset = pd.read_csv(test_files)\n",
    "train_dataset.columns = header\n",
    "test_dataset.columns = header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "494020 311028\n"
     ]
    }
   ],
   "source": [
    "train_dataset_size = train_dataset.shape[0]\n",
    "test_dataset_size = test_dataset.shape[0]\n",
    "train_dataset = pd.concat([train_dataset, test_dataset], axis=0)\n",
    "print train_dataset_size, test_dataset_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def labels_map(label):\n",
    "    label = str(label).split('.')[0]\n",
    "    if label == 'normal':\n",
    "        return 0\n",
    "    if label in ['ipsweep', 'mscan', 'nmap', 'portsweep', 'saint', 'satan']: #PROBE\n",
    "        return 1\n",
    "    if label in ['apache2', 'back', 'land', 'mailbomb', 'neptune', 'pod', 'processtable', 'smurf', 'teardrop', 'udpstorm']: #DOS\n",
    "        return 2\n",
    "    if label in ['buffer_overflow', 'httptunnel', 'loadmodule', 'perl', 'ps', 'rootkit', 'sqlattack', 'xterm']: #U2R\n",
    "        return 3\n",
    "    if label in ['ftp_write', 'guess_passwd', 'imap', 'multihop', 'named', 'phf', 'sendmail', 'snmpgetattack', 'snmpguess', 'spy', 'warezclient', 'warezmaster', 'worm', 'xlock', 'xsnoop']: #R2L\n",
    "        return 4\n",
    "    \n",
    "train_dataset['labels'] = train_dataset['labels'].apply(labels_map)\n",
    "labels_dummies = pd.get_dummies(train_dataset['labels'], prefix='label')\n",
    "train_dataset = pd.concat([train_dataset,labels_dummies], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>labels</th>\n",
       "      <th>label_0</th>\n",
       "      <th>label_1</th>\n",
       "      <th>label_2</th>\n",
       "      <th>label_3</th>\n",
       "      <th>label_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>239</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>235</td>\n",
       "      <td>1337</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>219</td>\n",
       "      <td>1337</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>217</td>\n",
       "      <td>2032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>217</td>\n",
       "      <td>2032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type service flag  src_bytes  dst_bytes  land  \\\n",
       "0         0           tcp    http   SF        239        486     0   \n",
       "1         0           tcp    http   SF        235       1337     0   \n",
       "2         0           tcp    http   SF        219       1337     0   \n",
       "3         0           tcp    http   SF        217       2032     0   \n",
       "4         0           tcp    http   SF        217       2032     0   \n",
       "\n",
       "   wrong_fragment  urgent  hot   ...     dst_host_serror_rate  \\\n",
       "0               0       0    0   ...                      0.0   \n",
       "1               0       0    0   ...                      0.0   \n",
       "2               0       0    0   ...                      0.0   \n",
       "3               0       0    0   ...                      0.0   \n",
       "4               0       0    0   ...                      0.0   \n",
       "\n",
       "   dst_host_srv_serror_rate  dst_host_rerror_rate  dst_host_srv_rerror_rate  \\\n",
       "0                       0.0                   0.0                       0.0   \n",
       "1                       0.0                   0.0                       0.0   \n",
       "2                       0.0                   0.0                       0.0   \n",
       "3                       0.0                   0.0                       0.0   \n",
       "4                       0.0                   0.0                       0.0   \n",
       "\n",
       "   labels  label_0  label_1  label_2  label_3  label_4  \n",
       "0       0        1        0        0        0        0  \n",
       "1       0        1        0        0        0        0  \n",
       "2       0        1        0        0        0        0  \n",
       "3       0        1        0        0        0        0  \n",
       "4       0        1        0        0        0        0  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "protocal_type_dummies = pd.get_dummies(train_dataset.protocol_type, prefix='protocol_type')\n",
    "service_dummies = pd.get_dummies(train_dataset.service, prefix='service')\n",
    "flag_dummies = pd.get_dummies(train_dataset.flag, prefix='flag')\n",
    "train_dataset = pd.concat([train_dataset, protocal_type_dummies, service_dummies, flag_dummies], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max1 = train_dataset.src_bytes.max(); min1 = train_dataset.src_bytes.min();\n",
    "max2 = train_dataset.dst_bytes.max(); min2 = train_dataset.dst_bytes.min();\n",
    "train_dataset['src_bytes_norm'] = (train_dataset.src_bytes - min1) / float(max1 - min1)\n",
    "train_dataset['dst_bytes_norm'] = (train_dataset.dst_bytes - min2) / float(max2 - min2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.drop(['protocol_type', 'service', 'flag', 'src_bytes', 'dst_bytes','labels'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.astype('float')\n",
    "# train_dataset = (train_dataset - train_dataset.min()) / (train_dataset.max() - train_dataset.min())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## preprocessing test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_train_dataset = train_dataset.iloc[train_dataset_size:, :].sample(n=200000)\n",
    "sub_test_dataset = train_dataset.iloc[:train_dataset_size, :].sample(n=50000)\n",
    "sub_train_labels = sub_train_dataset[['label_0', 'label_1', 'label_2', 'label_3', 'label_4']]\n",
    "sub_test_labels = sub_test_dataset[['label_0', 'label_1', 'label_2', 'label_3', 'label_4']]\n",
    "sub_train_dataset.drop(['label_0', 'label_1', 'label_2', 'label_3', 'label_4'], axis=1, inplace=True)\n",
    "sub_test_dataset.drop(['label_0', 'label_1', 'label_2', 'label_3', 'label_4'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_dataset = train_dataset.iloc[train_dataset_size:,:]\n",
    "train_dataset = train_dataset.iloc[:train_dataset_size, :]\n",
    "train_labels = train_dataset[['label_0', 'label_1', 'label_2', 'label_3', 'label_4']]\n",
    "test_labels = test_dataset[['label_0', 'label_1', 'label_2', 'label_3', 'label_4']]\n",
    "train_dataset.drop(['label_0', 'label_1', 'label_2', 'label_3', 'label_4'], axis=1, inplace=True)\n",
    "test_dataset.drop(['label_0', 'label_1', 'label_2', 'label_3', 'label_4'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(494020, 119) (494020, 5)\n",
      "(311028, 119) (311028, 5)\n"
     ]
    }
   ],
   "source": [
    "print train_dataset.shape, train_labels.shape\n",
    "print test_dataset.shape, test_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 119) (200000, 5)\n",
      "(50000, 119) (50000, 5)\n"
     ]
    }
   ],
   "source": [
    "print sub_train_dataset.shape, sub_train_labels.shape\n",
    "print sub_test_dataset.shape, sub_test_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>flag_S0</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "      <th>src_bytes_norm</th>\n",
       "      <th>dst_bytes_norm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.0000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.00000</td>\n",
       "      <td>2.000000e+05</td>\n",
       "      <td>200000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>17.169845</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000800</td>\n",
       "      <td>0.000040</td>\n",
       "      <td>0.014955</td>\n",
       "      <td>0.002455</td>\n",
       "      <td>0.171790</td>\n",
       "      <td>0.010575</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.002790</td>\n",
       "      <td>0.058225</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.000930</td>\n",
       "      <td>0.798005</td>\n",
       "      <td>0.00024</td>\n",
       "      <td>2.290441e-06</td>\n",
       "      <td>0.000146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>377.184612</td>\n",
       "      <td>0.004472</td>\n",
       "      <td>0.041223</td>\n",
       "      <td>0.007746</td>\n",
       "      <td>0.265088</td>\n",
       "      <td>0.050784</td>\n",
       "      <td>0.377199</td>\n",
       "      <td>1.662136</td>\n",
       "      <td>0.014316</td>\n",
       "      <td>0.005916</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002236</td>\n",
       "      <td>0.052747</td>\n",
       "      <td>0.234169</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.007071</td>\n",
       "      <td>0.030482</td>\n",
       "      <td>0.401490</td>\n",
       "      <td>0.01549</td>\n",
       "      <td>1.065741e-04</td>\n",
       "      <td>0.003359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.514331e-07</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>7.499542e-07</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.488371e-06</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>53771.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>611.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>4.563992e-02</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 119 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            duration           land  wrong_fragment         urgent  \\\n",
       "count  200000.000000  200000.000000   200000.000000  200000.000000   \n",
       "mean       17.169845       0.000020        0.000800       0.000040   \n",
       "std       377.184612       0.004472        0.041223       0.007746   \n",
       "min         0.000000       0.000000        0.000000       0.000000   \n",
       "25%         0.000000       0.000000        0.000000       0.000000   \n",
       "50%         0.000000       0.000000        0.000000       0.000000   \n",
       "75%         0.000000       0.000000        0.000000       0.000000   \n",
       "max     53771.000000       1.000000        3.000000       2.000000   \n",
       "\n",
       "                 hot  num_failed_logins      logged_in  num_compromised  \\\n",
       "count  200000.000000      200000.000000  200000.000000    200000.000000   \n",
       "mean        0.014955           0.002455       0.171790         0.010575   \n",
       "std         0.265088           0.050784       0.377199         1.662136   \n",
       "min         0.000000           0.000000       0.000000         0.000000   \n",
       "25%         0.000000           0.000000       0.000000         0.000000   \n",
       "50%         0.000000           0.000000       0.000000         0.000000   \n",
       "75%         0.000000           0.000000       0.000000         0.000000   \n",
       "max        30.000000           4.000000       1.000000       611.000000   \n",
       "\n",
       "          root_shell   su_attempted       ...          flag_RSTOS0  \\\n",
       "count  200000.000000  200000.000000       ...        200000.000000   \n",
       "mean        0.000205       0.000025       ...             0.000005   \n",
       "std         0.014316       0.005916       ...             0.002236   \n",
       "min         0.000000       0.000000       ...             0.000000   \n",
       "25%         0.000000       0.000000       ...             0.000000   \n",
       "50%         0.000000       0.000000       ...             0.000000   \n",
       "75%         0.000000       0.000000       ...             0.000000   \n",
       "max         1.000000       2.000000       ...             1.000000   \n",
       "\n",
       "           flag_RSTR        flag_S0      flag_S1        flag_S2  \\\n",
       "count  200000.000000  200000.000000  200000.0000  200000.000000   \n",
       "mean        0.002790       0.058225       0.0001       0.000050   \n",
       "std         0.052747       0.234169       0.0100       0.007071   \n",
       "min         0.000000       0.000000       0.0000       0.000000   \n",
       "25%         0.000000       0.000000       0.0000       0.000000   \n",
       "50%         0.000000       0.000000       0.0000       0.000000   \n",
       "75%         0.000000       0.000000       0.0000       0.000000   \n",
       "max         1.000000       1.000000       1.0000       1.000000   \n",
       "\n",
       "             flag_S3        flag_SF       flag_SH  src_bytes_norm  \\\n",
       "count  200000.000000  200000.000000  200000.00000    2.000000e+05   \n",
       "mean        0.000930       0.798005       0.00024    2.290441e-06   \n",
       "std         0.030482       0.401490       0.01549    1.065741e-04   \n",
       "min         0.000000       0.000000       0.00000    0.000000e+00   \n",
       "25%         0.000000       1.000000       0.00000    1.514331e-07   \n",
       "50%         0.000000       1.000000       0.00000    7.499542e-07   \n",
       "75%         0.000000       1.000000       0.00000    1.488371e-06   \n",
       "max         1.000000       1.000000       1.00000    4.563992e-02   \n",
       "\n",
       "       dst_bytes_norm  \n",
       "count   200000.000000  \n",
       "mean         0.000146  \n",
       "std          0.003359  \n",
       "min          0.000000  \n",
       "25%          0.000000  \n",
       "50%          0.000000  \n",
       "75%          0.000000  \n",
       "max          1.000000  \n",
       "\n",
       "[8 rows x 119 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_train_dataset.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model1: MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
    "          / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "feature_size = 119\n",
    "num_labels = 5\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data. For the training data, we use a placeholder that will be fed\n",
    "  # at run time with a training minibatch.\n",
    "  tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, feature_size))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "#   tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(sub_test_dataset.values[:,:])\n",
    "  \n",
    "  # Variables.\n",
    "  hidden_weights = tf.Variable(\n",
    "    tf.truncated_normal([feature_size, 1024]))\n",
    "  output_weights = tf.Variable(\n",
    "    tf.truncated_normal([1024, num_labels]))\n",
    "  weights = [hidden_weights, output_weights]\n",
    "  hidden_biases = tf.Variable(tf.zeros([1024]))\n",
    "  output_biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  biases = [hidden_biases, output_biases]\n",
    "  \n",
    "  # Training computation.\n",
    "  hidden_layer = tf.add(tf.matmul(tf_train_dataset, tf.cast(weights[0], tf.float32)), biases[0])\n",
    "  hidden_layer = tf.nn.relu(hidden_layer)\n",
    "  logits = tf.add(tf.matmul(hidden_layer, tf.cast(weights[1], tf.float32)), biases[1])\n",
    "#   logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf_train_labels))\n",
    "  l2Loss = tf.nn.l2_loss(weights[0]) + tf.nn.l2_loss(weights[1])\n",
    "  loss = loss + 5*l2Loss\n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.001).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "  # ValidPrediction\n",
    "#   temp1 = tf.add(tf.matmul(tf_valid_dataset, weights[0]), biases[0])\n",
    "#   hidden_temp = tf.nn.relu(temp1)\n",
    "#   temp2 = tf.add(tf.matmul(hidden_temp,weights[1]), biases[1])\n",
    "#   valid_prediction = tf.nn.softmax(temp2)\n",
    "\n",
    "  # TestPrediction\n",
    "  temp3 = tf.add(tf.matmul(tf.cast(tf_test_dataset, tf.float32), weights[0]), biases[0])\n",
    "  hidden_temp2 = tf.nn.relu(temp3)\n",
    "  temp4 = tf.add(tf.matmul(tf.cast(hidden_temp2, tf.float32), weights[1]), biases[1])\n",
    "  test_prediction = tf.nn.softmax(temp4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 253255.406250\n",
      "Minibatch accuracy: 0.9%\n",
      "Test accuracy: 79.5%\n",
      "Minibatch loss at step 300: 12116.498047\n",
      "Minibatch accuracy: 83.5%\n",
      "Test accuracy: 96.7%\n",
      "Minibatch loss at step 600: 596.876648\n",
      "Minibatch accuracy: 87.8%\n",
      "Test accuracy: 96.9%\n",
      "Minibatch loss at step 900: 29.804335\n",
      "Minibatch accuracy: 91.0%\n",
      "Test accuracy: 95.4%\n",
      "Minibatch loss at step 1200: 1.917058\n",
      "Minibatch accuracy: 90.7%\n",
      "Test accuracy: 95.3%\n",
      "Minibatch loss at step 1500: 0.561843\n",
      "Minibatch accuracy: 89.8%\n",
      "Test accuracy: 95.3%\n",
      "Minibatch loss at step 1800: 0.510866\n",
      "Minibatch accuracy: 89.2%\n",
      "Test accuracy: 95.3%\n",
      "Minibatch loss at step 2100: 0.513499\n",
      "Minibatch accuracy: 88.8%\n",
      "Test accuracy: 95.3%\n",
      "Minibatch loss at step 2400: 0.503703\n",
      "Minibatch accuracy: 89.8%\n",
      "Test accuracy: 95.4%\n",
      "Minibatch loss at step 2700: 0.436318\n",
      "Minibatch accuracy: 91.7%\n",
      "Test accuracy: 95.3%\n",
      "Minibatch loss at step 3000: 0.458622\n",
      "Minibatch accuracy: 90.2%\n",
      "Test accuracy: 95.3%\n",
      "Minibatch loss at step 3300: 0.448924\n",
      "Minibatch accuracy: 90.5%\n",
      "Test accuracy: 95.4%\n",
      "Minibatch loss at step 3600: 0.463286\n",
      "Minibatch accuracy: 90.4%\n",
      "Test accuracy: 95.4%\n",
      "Minibatch loss at step 3900: 0.498096\n",
      "Minibatch accuracy: 89.6%\n",
      "Test accuracy: 95.4%\n",
      "Minibatch loss at step 4200: 0.467283\n",
      "Minibatch accuracy: 90.4%\n",
      "Test accuracy: 95.7%\n",
      "Minibatch loss at step 4500: 0.482703\n",
      "Minibatch accuracy: 90.9%\n",
      "Test accuracy: 95.5%\n",
      "Minibatch loss at step 4800: 0.509639\n",
      "Minibatch accuracy: 89.1%\n",
      "Test accuracy: 95.7%\n",
      "Minibatch loss at step 5100: 0.475684\n",
      "Minibatch accuracy: 90.1%\n",
      "Test accuracy: 95.4%\n",
      "Minibatch loss at step 5400: 0.470665\n",
      "Minibatch accuracy: 90.9%\n",
      "Test accuracy: 95.6%\n",
      "Minibatch loss at step 5700: 0.501041\n",
      "Minibatch accuracy: 88.6%\n",
      "Test accuracy: 95.5%\n",
      "Minibatch loss at step 6000: 0.474377\n",
      "Minibatch accuracy: 89.3%\n",
      "Test accuracy: 95.4%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 6001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  #tf.initialize_all_variables().run()\n",
    "  tf.global_variables_initializer().run()\n",
    "  print(\"Initialized\")\n",
    "  for step in range(num_steps):\n",
    "    # Pick an offset within the training data, which has been randomized.\n",
    "    # Note: we could use better randomization across epochs.\n",
    "    offset = (step * batch_size) % (sub_train_labels.shape[0] - batch_size)\n",
    "    # Generate a minibatch.\n",
    "    batch_data = sub_train_dataset.iloc[offset:(offset + batch_size), :].values[:,:]\n",
    "    batch_labels = sub_train_labels.iloc[offset:(offset + batch_size), :].values[:,:]\n",
    "    # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "    # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "    # and the value is the numpy array to feed to it.\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 300 == 0):\n",
    "      print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "      print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "#       print(\"Validation accuracy: %.1f%%\" % accuracy(\n",
    "#         valid_prediction.eval(), valid_labels))\n",
    "      print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), sub_test_labels.values[:,:]))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "4_convolutions.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "86px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4.0,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}